var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/Communities';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/66xiaoqu';
var async = require('async');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');

if (!String.prototype.format) {
    String.prototype.format = function () {
        var args = arguments;
        return this.replace(/{(\d+)}/g, function (match, number) {
            return typeof args[number] != 'undefined'
                ? args[number]
                : match
                ;
        });
    };
}
var filteredWords = '小区,区,街,路,市,县,苑,花苑,花园,园,属,院,楼,巷,公寓,公房,大厦,大楼,别墅'.split(',')
function filterAddress(addr) {
    var new_addr = addr
    filteredWords.forEach(function (word) {
        new_addr = new_addr.replace(word, '')
        //console.log(word)
    })
    return new_addr
}

var dbconn = null;

function baiduCommunities(filters, callbackOuter) {
    // tag is '小区'
    var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=AvRqTglpaKICTZ5iwyn06KZj&page_size=20&page_num='
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=doqfhHvA5GjNLYnjjVFm9sL8&page_size=20&page_num='
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=ayGOiNeQNq6E9vSrbgeTrWbS&page_size=20&page_num='

    var dbCollection = 'baidu_md';
    var dbCollectionCommunities = 'baidu_md_communities'
    var s_collection = null;
    var t_collection = null;

    //var totalData = 0;
    //var filters = {"countyName" : "黄浦区"}
    //var filters = {'baidu': {'$in': [false, null]}}
    //var filters = {$or: [{'latitude': null}, {'longitude': null}]}
    //var filters = {ZDZQC:"河北 石家庄 行唐县"}

    function getDistricts(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                if(!err){
                    dbconn = db
                    assert.equal(null, err);
                    //console.log("Connected correctly to server");
                    t_collection = db.collection(dbCollectionCommunities);
                    s_collection = db.collection(dbCollection);
                    s_collection.count(filters, function (err, count) {
                        //totalData = count
                        //console.log("Total data %s: %s", dbCollection, totalData)
                        if (count == 0){
                            console.log("NO DATA IN " + dbCollection + " FOR",filters)
                            dbconn.close()
                            callbackOuter()
                        }
                        else
                            callback()
                    });
                }else{
                    console.log(err)
                    callbackOuter()
                }
            });
    };

    function run() {
        var manifest = {
            target: {
                url: urlTemp,
                ranges: {
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            extractors: [{
                name: 'details',
                root: '',
                fields: {}
            }]
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
        });

        function init(callback) {
            callback(null, 'init');
        }

        var items = [];

        function findCommids(db, callback) {
            s_collection.find(filters).toArray(
                function (err, docs) {
                    callback(docs);
                });
        };

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    items = docs;
                    callback(null, 'getIds');
                }
            );
        }

        function insertDocuments(doc, callback) {
            var new_doc = {}
            for (var f in doc) {
                if (f == 'name') {
                    new_doc['communityName'] = doc['name']
                }
                else if (f == 'address') {
                    var addr = doc['address']
                    var city = doc.city.replace('市','')
                    var county = doc.county.replace('区','').replace('县','')
                    new_doc['communityAddress'] = addr
                    //if(addr.indexOf('街') > -1){
                    //    new_doc['road'] = addr.substr(0,addr.indexOf('街'))
                    //    new_doc['road'] = new_doc['road'].replace(city+'市','').replace(city,'').replace(county+'区','').replace(county+'县','').replace(county,'')
                    //}else if (addr.indexOf('路') > -1){
                    //    new_doc['road'] = addr.substr(0,addr.indexOf('路'))
                    //    new_doc['road'] = new_doc['road'].replace(city+'市','').replace(city,'').replace(county+'区','').replace(county+'县','').replace(county,'')
                    //}
                }
                else if (f == 'location') {
                    new_doc['longitude'] = doc[f]['lng']
                    new_doc['latitude'] = doc[f]['lat']
                } else if (f == 'detail_info') {
                    for (var df in doc[f]) {
                        new_doc[df] = doc[f][df]
                    }
                } else {
                    new_doc[f] = doc[f]
                }
            }
            if('price' in new_doc){
                new_doc.price = parseInt(new_doc.price)
            }
            var updateKey = {}
            if('street_id' in new_doc){
                updateKey['street_id'] = new_doc['street_id']
            }
            t_collection.updateOne(updateKey, new_doc, {upsert:true}, function (err, result) {
                console.log("Upserted %s (%s) into collection %s", new_doc.communityName, new_doc.city + ' ' +new_doc.county, dbCollectionCommunities);
                callback()
            });
        };

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(items, function (item, callbackInner) {

                    var query = item.cityName + ' ' + item.countyName
                    console.log("\n*****STARTING %s*****",query)
                    var url = urlTemp.format(encodeURIComponent(query), encodeURIComponent((query)))

                    var pageNo = 1

                    async.whilst(function(){return pageNo > 0},function(next){
                        manifest.target.url = url + pageNo
                        console.log(manifest.target.url)

                        var job1 = new Crawler(manifest);
                        //console.log(job1)

                        job1.on('data', function (data) {
                            //console.log(data)
                            var $ = cheerio.load(data.request.body);
                            try{
                                var dataRoot = JSON.parse($._root.children[0].data)
                            }catch(e){
                                console.log(e)
                                callbackInner();
                                pageNo = -1
                            }
                            //console.log(dataRoot)
                            if (parseInt(dataRoot.status) == 302) {
                                console.log("YOU HAVE BEEN BLOCKED BY BAIDU!")
                                process.exit()
                            }
                            else if (parseInt(dataRoot.status) == 240) {
                                console.log("INVALID DEVELOPER KEY!")
                                process.exit()
                            }
                            else if (parseInt(dataRoot.status) == 210) {
                                console.log("IP NOT ALLOWED FOR THIS DEVELOPER KEY!")
                                process.exit()
                            }
                            var data = dataRoot.results

                            var dataLen = data.length
                            if (dataLen == 0){  // no results left
                                console.log("\nNo results left for %s!", query)
                                callbackInner();
                                pageNo = -1
                            }else {
                                var counter = 0
                                async.eachSeries(data, function (doc, callbackInner2) {
                                    doc.city = item.cityName
                                    doc.cityId = item.cityId
                                    doc.county = item.countyName
                                    doc.countyId = item._id

                                    insertDocuments(doc, function () {
                                        counter++
                                        callbackInner2()
                                        if(counter == dataLen){
                                            pageNo++
                                            next()
                                        }
                                    })
                                })
                            }
                        });

                        job1.on('error', function (err) {
                            throw err;
                        });
                        job1.on('end', function () {
                            //console.log('The job is done: %d', pageNo);
                            delete job1;
                        });
                        job1.start();
                    },function(err){})

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            //console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);
            });
    }

    getDistricts(run);
}

//var filters = {countyName:{$nin:["孟津县","偃师市","嵩县","禹王台区","顺河区","金明区","开封县","杞县","金水区","经济开发区","惠济区","新密市","沁源县","黎城县","襄垣县","盐湖区","闻喜县","临猗县","稷山县","平陆县","永济市","洪洞县","安泽县","霍州市","中阳县","孝义市","定襄县","繁峙县","代县","榆次区","平遥县","灵石县","太谷县","丰台区","昔阳县","古冶区","介休市","祁县","忻府区","沙河市","渝中区","泽州县","沁水县","平定县","金山区","矿区","郊区","阳高县","尖草坪区","小店区","新安县","清徐县","伊川县","房山区","古交市","兰考县","阳曲县","桃城区","中牟县","故城县","晋源区","枣强县","景县","绛县","三河市","石景山区","广阳区","开发区","宣化区","大厂县","武安市","黄骅市","任丘市","古县","蠡县","吴桥县","侯马市","长寿区","南市区","东光县","深泽县","肃宁县","青县","献县","新郑市","迁安市","开平区","冀州市","玉田县","宜阳县","盂县","滦南县","海港区","巩义市","大足区","丰南区","青龙县","尧都区","双桥区","安平县","兴隆县","左云县","霸州市","南宫市","大同县","抚宁县","怀来县","怀安县","昌黎县","中原区","宽城县","张北县","安次区","北戴河区","徐水县","文水县","峰峰矿区","开县","朔城区","浑源县","阳原县","二七区","垣曲县","北部新区","高碑店市","安新县","丰润区","奉贤区","蔚县","涿州市","顺平县","怀仁县","和顺县","应县","永清县","曲阳县","河间市","新市区","阜平县","清苑县","武清区","灵丘县","满城县","高新技术开发区","南岸区","北市区","昌平区","雄县","邢台县","易县","宁晋县","深州市","巨鹿县","山海关区","丰宁县","左权县","蓟县","壶关县","徐汇区","东城区","清河县","南皮县","交城县","无极县","复兴区","芮城县","万柏林区","津南区","固安县","翼城县","杨浦区","垫江县","路南区","邯山区","香河县","平泉县","黔江区","新华区","唐县","高邑县","丛台区","延庆县","长治县","大名县","文安县","隆化县","馆陶县","鹿泉市","栾川县","密云县","曲周县","通州区","洛宁县","井陉矿区","河曲县","邯郸县","朝阳区","高阳县","崇明县","行唐县","元氏县","原平市","晋州市","奉节县","武隆县","滦县","沽源县","隆尧县","虹口区","阳城县","云阳县","襄汾县","双滦区","九龙坡区","遵化市","涉县","江北区","郑东新区","乐亭县","普陀区","顺义区","辛集市","大港区","北碚区","桥东区","下花园区","闸北区","平谷区","巴南区","正定县","桥西区","松江区","合川区","登封市","永年县","涞水县","酉阳县","静安区","怀柔区","荣昌县","新河县","忠县","赵县","南郊区","璧山县","路北区","容城县","巫溪县","威县","涪陵区","石柱县","潞城市","闵行区","永川区","丰都县","定兴县","大城县","河东区","成安县","宝坻区","长安区","渝北区","涿鹿县","青浦区","迁西县","海淀区","南川区","万全县","石河子市","梁平县","泊头市","卢龙县","汉沽区","万州区","北辰区","大渡口区","西城区","盐山县","静海县","崇礼县","承德县","西青区","定州市","宝山区","南开区","红桥区","上街区","东丽区","运河区","河津市","长宁区","望都县","尉氏县","和平区","离石区","内丘县","围场县","塘沽区","磁县","汾阳市","綦江区","宁河县","高新区","铜梁县","管城区","江津区","河北区","灵寿县","大兴区","沙坪坝区","黄浦区","迎泽区","门头沟区","安国市","栾城县","潼南县","高平市","临漳县","吉利区","荥阳市","魏县","平山县","长子县","藁城市","涞源县","新乐市","浦东新区","裕华区","嘉定区","杏花岭区","河西区"]}}
//var filters = {'countyName':{'$exists':true}}
var filters = {'cityName':'伊犁州'}
baiduCommunities(filters)

var exports = module.exports
exports.baiduCommunities = baiduCommunities